#!/usr/bin/env python
# encoding=utf-8

"""
深圳证券交易所-监管措施
http://www.szse.cn/main/disclosure/jgxxgk/jgcs/
"""

import sys
from random import random

from pymongo import MongoClient
from scpy.logger import get_logger
from xtls.basecrawler import BaseCrawler
from xtls.codehelper import no_exception
from xtls.timeparser import parse_time, now
from xtls.util import BeautifulSoup, sha1

from util import get_mongo_conf

reload(sys)
sys.setdefaultencoding('utf-8')

__author__ = 'xu.du'
logger = get_logger(__file__)

LIST_URL = 'http://www.szse.cn/main/disclosure/jgxxgk/jgcs/'
AJAX_URL = 'http://www.szse.cn/szseWeb/FrontController.szse?randnum={rand}'
THRESHOLD = 19
HOST, PORT = get_mongo_conf()
CONN = MongoClient(HOST, PORT)


class SzseCrawler(BaseCrawler):
    def __init__(self):
        super(SzseCrawler, self).__init__(status=[0, 0, 0])  # success=0, saved=0, error=0,
        self._request.headers.update({'Host': 'www.szse.cn'})
        logger.info('init Szse Crawler done.')

    @no_exception(on_exception=False, logger=logger)
    def parse_table(self, soup):
        table = soup.find('table', attrs={'id': 'REPORTID_tab1'})
        trs = table.find_all('tr', attrs={'class': 'cls-data-tr'})
        for index, tr in enumerate(trs):
            print index
            tds = [td.getText().strip() for td in tr.find_all('td')]
            if len(tds) < 6:
                continue
            result = self.save({
                'stockCode': tds[0],  # 股票代码
                'companyShortName': tds[1],  # 公司简称
                'regulatoryMeasuresTime': parse_time(tds[2]),  # 采取监管措施日期
                'regulatoryMeasures': tds[3],  # 监管措施
                'involvedSubject': tds[4],  # 涉及事由
                'involvedObject': tds[5],  # 涉及对象
                'sourceStockExchange': 'szse',
                'remarks': ''  # 备注
            })
            self.status[result] += 1

        if self.status[1] > THRESHOLD or self.status[2] > THRESHOLD:
            return False

        next_page_param = soup.find('input', attrs={'class': 'cls-navigate-next'})['onclick'].strip()
        next_page_param = next_page_param[next_page_param.find('FrontController.szse?') + 21:-2]
        next_page_param = dict(item.split('=') for item in next_page_param.split('&'))
        return next_page_param

    @no_exception(on_exception=2, logger=logger)
    def save(self, data):
        data['_id'] = sha1(str(data.values()))
        logger.info('save item : %s' % data['_id'])
        data['updateTime'] = now()
        result = CONN['crawler_company_all']['regulatoryMeasures'].find_one_and_update(
            filter={'_id': data['_id']},
            update={'$set': data}, upsert=True)
        if result:
            return 1
        return 0

    def run(self):
        logger.info('first page.')
        html = self.get(LIST_URL)
        soup = BeautifulSoup(html)
        next_page_post_data = self.parse_table(soup)
        while next_page_post_data:
            logger.info('now post data : %s' % next_page_post_data)
            html = self.post(AJAX_URL.format(rand=random()), data=next_page_post_data)
            soup = BeautifulSoup(html)
            next_page_post_data = self.parse_table(soup)
            # print next_page_post_data
            # break


def main():
    SzseCrawler().run()


if __name__ == '__main__':
    main()
